read_sacct_out <- function(filename,nodes_desc=NULL,extract_node_list=FALSE){
    cat("Read: ",filename,"\n")
  
    if(grepl(".zst", filename, fixed=TRUE)){
      slurm_log <- data.table::fread(cmd=paste("zstdcat", filename),sep="|",header=TRUE)
    } else {
      slurm_log <- data.table::fread(filename,sep="|",header=TRUE)
    }
    
    #for(col in c("Submit","Eligible","Start","End","Elapsed","Timelimit",
    #             "Cluster","Partition","Account","Group","User", "ExitCode","State","QOS")){
        #cat(paste0(col,"S=",col,",\n"))
        #cat(paste0(col,"S,"))
    #}
    slurm_log <-  dplyr::rename(slurm_log,
        JobId=JobID,
        JobIdRaw=JobIDRaw,
        NodeCount=NNodes,
        SubmitS=Submit,
        EligibleS=Eligible,
        StartS=Start,
        EndS=End,
        ElapsedS=Elapsed,
        TimelimitS=Timelimit,
        ClusterS=Cluster,
        PartitionS=Partition,
        AccountS=Account,
        GroupS=Group,
        UserS=User,
        ExitCodeS=ExitCode,
        StateS=State,
        QOSS=QOS
    )
    #convert job names
    slurm_log$JobRecID <- as.integer(str_replace(slurm_log$JobName, "jobid_",""))
    
    #convert to proper format
    for(col in c("Submit","Eligible","Start","End")){
        slurm_log[[col]] <- as.POSIXct(slurm_log[[paste0(col,"S")]],format = "%Y-%m-%dT%H:%M:%S")
    }
    
    #duration
    for(col in c("Elapsed","Timelimit")){
        slurm_log[,col] <- extract_slurm_period(slurm_log[[paste0(col,"S")]])
    }
    
    #factor
    for(col in c("Cluster","Partition","Account","Group","User", "ExitCode","State","QOS")){
        slurm_log[,col] <- factor(slurm_log[[paste0(col,"S")]])
    }
    
    #state
    slurm_log$StateS <- as.character(slurm_log$StateS)
    slurm_log$StateS[grepl("CANCELLED",slurm_log$StateS)] <- "CANCELLED"
    slurm_log$State <- as.factor(slurm_log$StateS)
    
    #extract node list
    if(extract_node_list==TRUE){
        #python.load(file.path(rutil_dir,"..","src","hostlist.py"))
        #slurm_log$NodeListFull <- python.call("expand_hostlists_to_str",slurm_log$NodeList)
        slurm_log$NodeListFull <- expand_hostlists_to_list(slurm_log$NodeList)
    }

    #convert memory
    slurm_log$ReqMem[slurm_log$ReqMem=="0n"] <- "0Mn"
    reqmem <- stringr::str_match_all(slurm_log$ReqMem, "([\\.0-9]+)([MG])([nc]?)")
    
    reqmem_size <- sapply(reqmem,function(r){
        as.integer(r[[2]])
    })
    reqmem_unit <- sapply(reqmem,function(r)r[[3]])
    reqmem_perwhat <- sapply(reqmem,function(r)r[[4]])
    #convert to MB
    reqmem_size[reqmem_unit=="G"] <- reqmem_size[reqmem_unit=="G"]*1024
    
    slurm_log$ReqMemSize <- reqmem_size
    slurm_log$ReqMemPerNode <- reqmem_perwhat=="n" || reqmem_perwhat==""
    
    slurm_log$ReqMem <- NULL
    
    #set proper NA
    #slurm_log$ReqGRES[slurm_log$ReqGRES==""] <- NA
    if(!is.null(nodes_desc)){
      nr <- max(sapply(nodes_desc,function(n){length(n$Nodes)}))
      
      nodes_mat <- sapply(nodes_desc,function(n){c(n$Nodes,rep(NA,nr-length(n$Nodes)))})
      
      #assing nodes
      nodes_types_used <- sapply(slurm_log$NodeListFull,function(nodes){
        apply(nodes_mat,2,function(v){length(intersect(v,nodes))})
      })
      
      slurm_log <- cbind(slurm_log,t(nodes_types_used))
    }
    
    slurm_log$SubmitTS <- as.integer(slurm_log$Submit)
    slurm_log$StartTS <- as.integer(slurm_log$Start)
    slurm_log$EndTS <- as.integer(slurm_log$End)
    
    
    slurm_log$WaitHours <- as.integer(slurm_log$Start-slurm_log$Submit)/3600.0
    slurm_log$WaitHours[slurm_log$WaitHours<0.0] <- slurm_log$WaitHours[slurm_log$WaitHours<0.0]+1
    
    slurm_log$WallHours <- as.integer(slurm_log$Elapsed)/3600.0
    slurm_log$NodeHours <- slurm_log$WallHours*slurm_log$NodeCount
    
    #shift 0 value for log scales
    slurm_log$WaitHours4log <- slurm_log$WaitHours
    slurm_log$WaitHours4log[slurm_log$WaitHours4log<1/60]<-1/60
    #shift 0 value for log scales
    slurm_log$WallHours4log <- slurm_log$WallHours
    slurm_log$WallHours4log[slurm_log$WallHours4log<1/60]<-1/60
    
    slurm_log <- dplyr::arrange(slurm_log,SubmitTS)%>%
        dplyr::select(-c(SubmitS,EligibleS,StartS,EndS,ElapsedS,TimelimitS,ClusterS,
                         PartitionS,AccountS,GroupS,UserS,ExitCodeS,StateS,QOSS))
    
    min_time <- min(min(slurm_log$SubmitTS, na.rm=TRUE),min(slurm_log$StartTS, na.rm=TRUE),min(slurm_log$EndTS, na.rm=TRUE), na.rm=TRUE)
    
    slurm_log$SubmitTime <- slurm_log$SubmitTS - min_time
    slurm_log$StartTime <- slurm_log$StartTS - min_time
    slurm_log$EndTime <- slurm_log$EndTS - min_time
    slurm_log$WallTime <- slurm_log$EndTime - slurm_log$StartTime
    slurm_log$WaitTime <- slurm_log$StartTime - slurm_log$SubmitTime
    
    slurm_log <- relocate(slurm_log, JobRecID, SubmitTime, StartTime, EndTime, WallTime, WaitTime, .before = "JobId")
    
    return(slurm_log)
}

read_sacct_out_multiple <- function(slurm_mode, results_root_dir, dtstart_list, run_id_list, sacct_out="slurm_acct.out") {
  result_list <- list()
  for(dtstart in dtstart_list) {
    for(run_id in run_id_list) {
      m_result_root_dir <- path.expand(file.path(results_root_dir, paste0("dtstart_", dtstart, "_", run_id)))
      m_sacct_out_filename <- file.path(m_result_root_dir, sacct_out)
      
      if(!dir.exists(m_result_root_dir)) {
          warning(sprintf("Directory %s does not exists!", m_result_root_dir))
          return(NULL);
      }
      if(!file.exists(m_sacct_out_filename)) {
          if (file.exists(paste0(m_sacct_out_filename,".zst"))) {
              m_sacct_out_filename <- paste0(m_sacct_out_filename,".zst")
          } else {
              warning(sprintf("File %s does not exists!", m_sacct_out_filename))
              return(NULL);
          }
      }
      m_sacct_out <- read_sacct_out(m_sacct_out_filename)
      m_sacct_out$slurm_mode <- slurm_mode
      m_sacct_out$dtstart <- dtstart
      m_sacct_out$run_id <- run_id
      
      m_sacct_out <- relocate(m_sacct_out,slurm_mode,dtstart,run_id, .before = "JobRecID")
      result_list[[length(result_list)+1]] <- m_sacct_out
      
    }
  }
  return(data.table::rbindlist(result_list))
}

#JobName
# ../../../slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/
sacct <- bind_rows(
  read_sacct_out_multiple("bumblebee_0_tarball", path.expand("~/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1:2),
  read_sacct_out_multiple("bumblebee_1_351d4", path.expand("~/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1:2),
  read_sacct_out_multiple("bumblebee_2_f588c", path.expand("~/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1:2),
  read_sacct_out_multiple("bumblebee_3_f588c", path.expand("~/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1:2),
  read_sacct_out_multiple("bumblebee_4_f588c", path.expand("~/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1:2),
  read_sacct_out_multiple("bumblebee_41_f588c", path.expand("~/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1),
  read_sacct_out_multiple("bumblebee_92_f588c", path.expand("~/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small"), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1)
)
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_58_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_58_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_59_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_59_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_68_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_68_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_79_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_79_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_89_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_89_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_99_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_99_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_105_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_105_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_114_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_114_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_126_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_126_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_146_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_21.08.04_tarball/small__ref/dtstart_146_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_58_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_58_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_59_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_59_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_68_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_68_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_79_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_79_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_89_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_89_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_99_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_99_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_105_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_105_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_114_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_114_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_126_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_126_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_146_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_146_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_2/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_3_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_2/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_4_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_1/slurm_acct.out 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_58_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_59_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_68_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_79_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_89_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_99_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_105_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_114_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_126_1/slurm_acct.out.zst 
## Read:  /home/nikolays/slurm_sim_ws/slurm_model/micro3simdev/results/bumblebee_2_f588c59c7a52c00c95f8c84c94e9122048c0acd0/small/dtstart_146_1/slurm_acct.out.zst
#sacct$rid <- sacct$run_id + sacct$dtstart*100
#sacct
#read_sacct_out(path.expand("~/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small/dtstart_58_1/slurm_acct.out"))
sacct
##                slurm_mode dtstart run_id JobRecID SubmitTime StartTime EndTime
##    1: bumblebee_0_tarball      58      1     1001          0         1       1
##    2: bumblebee_0_tarball      58      1     1002          1         3      63
##    3: bumblebee_0_tarball      58      1     1003          2         3       8
##    4: bumblebee_0_tarball      58      1     1004         16        17      38
##    5: bumblebee_0_tarball      58      1     1005         19        40      42
##   ---                                                                         
## 2396:  bumblebee_92_f588c     146      1     1016         40        89     115
## 2397:  bumblebee_92_f588c     146      1     1017         42        86      87
## 2398:  bumblebee_92_f588c     146      1     1018         42        55     116
## 2399:  bumblebee_92_f588c     146      1     1019         43        55      89
## 2400:  bumblebee_92_f588c     146      1     1020         43       115     129
##       WallTime WaitTime JobId JobIdRaw GID   UID NodeCount NCPUS ReqCPUS
##    1:        0        1  1000     1000 100 10005         1    12      12
##    2:       60        2  1001     1001 100 10001         1     1       1
##    3:        5        1  1002     1002 100 10004         1     1       1
##    4:       21        1  1003     1003 100 10003         2    24      24
##    5:        2       21  1004     1004 100 10005         1    12      12
##   ---                                                                   
## 2396:       26       49  1015     1015   0 10001         1     2       2
## 2397:        1       44  1016     1016   0 10001         4    48      48
## 2398:       61       13  1017     1017   0 10003         1    12      12
## 2399:       34       12  1018     1018   0 10004         1    12      12
## 2400:       14       72  1019     1019   0 10001         1     1       1
##                                    ReqTRES NodeList    JobName NTasks
##    1:  billing=12,cpu=12,mem=33600M,node=1       b1 jobid_1001     NA
##    2:     billing=1,cpu=1,mem=2800M,node=1       n1 jobid_1002     NA
##    3:   billing=1,cpu=1,mem=500000M,node=1       b1 jobid_1003     NA
##    4:  billing=24,cpu=24,mem=67200M,node=1    b1,g1 jobid_1004     NA
##    5: billing=12,cpu=12,mem=500000M,node=1       b1 jobid_1005     NA
##   ---                                                                
## 2396:     billing=2,cpu=2,mem=5600M,node=1       g1 jobid_1016     NA
## 2397: billing=48,cpu=48,mem=134400M,node=1   n[1-4] jobid_1017     NA
## 2398:  billing=12,cpu=12,mem=33600M,node=1       m4 jobid_1018     NA
## 2399:  billing=12,cpu=12,mem=33600M,node=1       g1 jobid_1019     NA
## 2400:     billing=1,cpu=1,mem=2800M,node=1       n2 jobid_1020     NA
##                    Submit            Eligible               Start
##    1: 2021-12-14 17:49:04 2021-12-14 17:49:04 2021-12-14 17:49:05
##    2: 2021-12-14 17:49:05 2021-12-14 17:49:05 2021-12-14 17:49:07
##    3: 2021-12-14 17:49:06 2021-12-14 17:49:06 2021-12-14 17:49:07
##    4: 2021-12-14 17:49:20 2021-12-14 17:49:20 2021-12-14 17:49:21
##    5: 2021-12-14 17:49:23 2021-12-14 17:49:23 2021-12-14 17:49:44
##   ---                                                            
## 2396: 2022-01-26 05:48:18 2022-01-26 05:48:18 2022-01-26 05:49:07
## 2397: 2022-01-26 05:48:20 2022-01-26 05:48:20 2022-01-26 05:49:04
## 2398: 2022-01-26 05:48:20 2022-01-26 05:48:20 2022-01-26 05:48:33
## 2399: 2022-01-26 05:48:21 2022-01-26 05:48:21 2022-01-26 05:48:33
## 2400: 2022-01-26 05:48:21 2022-01-26 05:48:21 2022-01-26 05:49:33
##                       End             Elapsed        Timelimit Cluster
##    1: 2021-12-14 17:49:05                  0s 60s (~1 minutes)   micro
##    2: 2021-12-14 17:50:07    60s (~1 minutes) 60s (~1 minutes)   micro
##    3: 2021-12-14 17:49:12                  5s 60s (~1 minutes)   micro
##    4: 2021-12-14 17:49:42                 21s 60s (~1 minutes)   micro
##    5: 2021-12-14 17:49:46                  2s 60s (~1 minutes)   micro
##   ---                                                                 
## 2396: 2022-01-26 05:49:33                 26s 60s (~1 minutes)   micro
## 2397: 2022-01-26 05:49:05                  1s 60s (~1 minutes)   micro
## 2398: 2022-01-26 05:49:34 61s (~1.02 minutes) 60s (~1 minutes)   micro
## 2399: 2022-01-26 05:49:07                 34s 60s (~1 minutes)   micro
## 2400: 2022-01-26 05:49:47                 14s 60s (~1 minutes)   micro
##       Partition  Account Group  User ExitCode     State    QOS ReqMemSize
##    1:    normal account2 users user5      0:0 COMPLETED normal      33600
##    2:    normal account1 users user1      0:0   TIMEOUT normal       2800
##    3:    normal account2 users user4      0:0 COMPLETED normal     500000
##    4:    normal account1 users user3      0:0 COMPLETED normal      67200
##    5:    normal account2 users user5      0:0 COMPLETED normal     500000
##   ---                                                                    
## 2396:    normal account1  root user1      0:0 COMPLETED normal       5600
## 2397:    normal account1  root user1      0:0 COMPLETED normal     134400
## 2398:    normal account1  root user3      0:0   TIMEOUT normal      33600
## 2399:    normal account2  root user4      0:0 COMPLETED normal      33600
## 2400:    normal account1  root user1      0:0 COMPLETED normal       2800
##       ReqMemPerNode   SubmitTS    StartTS      EndTS    WaitHours    WallHours
##    1:          TRUE 1639522144 1639522145 1639522145 0.0002777778 0.0000000000
##    2:          TRUE 1639522145 1639522147 1639522207 0.0005555556 0.0166666667
##    3:          TRUE 1639522146 1639522147 1639522152 0.0002777778 0.0013888889
##    4:          TRUE 1639522160 1639522161 1639522182 0.0002777778 0.0058333333
##    5:          TRUE 1639522163 1639522184 1639522186 0.0058333333 0.0005555556
##   ---                                                                         
## 2396:          TRUE 1643194098 1643194147 1643194173 0.0136111111 0.0072222222
## 2397:          TRUE 1643194100 1643194144 1643194145 0.0122222222 0.0002777778
## 2398:          TRUE 1643194100 1643194113 1643194174 0.0036111111 0.0169444444
## 2399:          TRUE 1643194101 1643194113 1643194147 0.0033333333 0.0094444444
## 2400:          TRUE 1643194101 1643194173 1643194187 0.0200000000 0.0038888889
##          NodeHours WaitHours4log WallHours4log
##    1: 0.0000000000    0.01666667    0.01666667
##    2: 0.0166666667    0.01666667    0.01666667
##    3: 0.0013888889    0.01666667    0.01666667
##    4: 0.0116666667    0.01666667    0.01666667
##    5: 0.0005555556    0.01666667    0.01666667
##   ---                                         
## 2396: 0.0072222222    0.01666667    0.01666667
## 2397: 0.0011111111    0.01666667    0.01666667
## 2398: 0.0169444444    0.01666667    0.01694444
## 2399: 0.0094444444    0.01666667    0.01666667
## 2400: 0.0038888889    0.02000000    0.01666667
jobs_summary <- sacct %>% group_by(slurm_mode,JobRecID) %>% summarise(
  SubmitTimeMean=mean(SubmitTime), SubmitTimeSD=sd(SubmitTime),
  StartTimeMean=mean(StartTime), StartTimeSD=sd(StartTime),
  WaitTimeMean=mean(WaitTime), WaitTimeSD=sd(WaitTime),
  WallTimeMean=mean(WallTime), WallTimeSD=sd(WallTime),
  .groups="drop_last"
) %>% arrange(JobRecID,slurm_mode)
jobs_summary
## # A tibble: 140 x 10
## # Groups:   slurm_mode [7]
##    slurm_mode JobRecID SubmitTimeMean SubmitTimeSD StartTimeMean StartTimeSD
##    <chr>         <int>          <dbl>        <dbl>         <dbl>       <dbl>
##  1 bumblebee…     1001              0            0          0.25       0.444
##  2 bumblebee…     1001              0            0          0.2        0.410
##  3 bumblebee…     1001              0            0          1.2        0.410
##  4 bumblebee…     1001              0            0          0.65       0.489
##  5 bumblebee…     1001              0            0          0.75       0.444
##  6 bumblebee…     1001              0            0          0.7        0.483
##  7 bumblebee…     1001              0            0          1.2        0.422
##  8 bumblebee…     1002              1            0          3.15       0.813
##  9 bumblebee…     1002              1            0          2.95       0.826
## 10 bumblebee…     1002              1            0          1.2        0.410
## # … with 130 more rows, and 4 more variables: WaitTimeMean <dbl>,
## #   WaitTimeSD <dbl>, WallTimeMean <dbl>, WallTimeSD <dbl>

bumblebee_0_tarball - build from official tarball bumblebee_1_351d4 - build from git using tag corresponding to above tarball, increased time accuracy in logs bumblebee_2_f588c - first attempt to feed jobs from slurmctrld bumblebee_3_f588c - same as 2 but feeding as in 1 bumblebee_4_f588c - same as 2 but feeding as in 1, also read trace but dont loop

read_sacct_out_multiple(“bumblebee_1_351d4”, path.expand(“~/slurm_sim_ws/slurm_model_archive/micro3/bumblebee_1_351d4b6c2870284078531f082758f11116f0996d/small”), c(58, 59, 68, 79, 89, 99, 105, 114, 126, 146), 1:2), read_sacct_out_multiple(“bumblebee_2_f588c”

plot_grid(
  ggplot(jobs_summary, aes(x=JobRecID,y=StartTimeMean, fill=slurm_mode)) + 
    geom_bar(stat="identity", position=position_dodge()) + 
    theme(legend.position="bottom"),
    ggplot(jobs_summary, aes(x=JobRecID,y=StartTimeSD, fill=slurm_mode)) + 
    geom_bar(stat="identity", position=position_dodge()) + 
    theme(legend.position="bottom"),
  ncol=1)

ggplot(sacct, aes(x=factor(JobRecID),y=StartTime, color=slurm_mode)) + 
    geom_boxplot(position=position_dodge()) + 
    theme(legend.position="bottom")

#ggplotly(p)

Start Time Boxplot

plot_ly(sacct, x = ~factor(JobRecID), y = ~StartTime, color = ~slurm_mode, type = "box") %>% 
  layout(boxmode = "group", legend = list(orientation = 'h', y=-0.2))
## Warning: 'layout' objects don't have these attributes: 'boxmode'
## Valid attributes include:
## 'font', 'title', 'uniformtext', 'autosize', 'width', 'height', 'margin', 'computed', 'paper_bgcolor', 'plot_bgcolor', 'separators', 'hidesources', 'showlegend', 'colorway', 'datarevision', 'uirevision', 'editrevision', 'selectionrevision', 'template', 'modebar', 'newshape', 'activeshape', 'meta', 'transition', '_deprecated', 'clickmode', 'dragmode', 'hovermode', 'hoverdistance', 'spikedistance', 'hoverlabel', 'selectdirection', 'grid', 'calendar', 'xaxis', 'yaxis', 'ternary', 'scene', 'geo', 'mapbox', 'polar', 'radialaxis', 'angularaxis', 'direction', 'orientation', 'editType', 'legend', 'annotations', 'shapes', 'images', 'updatemenus', 'sliders', 'colorscale', 'coloraxis', 'metasrc', 'barmode', 'bargap', 'mapType'
data(iris)

fig <- plot_ly()%>%
  add_trace(data = iris, x = ~Sepal.Width, y = ~Sepal.Length, color = ~Species, 
               type = "scatter", mode = "markers")%>%
  layout(title="Using The add_trace() method With A Plotly Figure",  legend=list(title=list(text='species')),
         plot_bgcolor='#e5ecf6', 
         xaxis = list( 
           zerolinecolor = '#ffff', 
           zerolinewidth = 2, 
           gridcolor = 'ffff'), 
         yaxis = list( 
           zerolinecolor = '#ffff', 
           zerolinewidth = 2, 
           gridcolor = 'ffff'))%>%
  add_trace(x = c(2, 4), y = c(4, 8), type = "scatter", mode = "lines", line = list(color = 'grey')
            , showlegend = FALSE)
fig